'''
https://movie.douban.com/subject/34780991/comments?start=20&limit=20&status=P&sort=new_score
https://movie.douban.com/subject/34780991/comments?start=40&limit=20&status=P&sort=new_score
https://movie.douban.com/subject/34780991/comments?start=60&limit=20&status=P&sort=new_score
'''

import logging
import requests
from bs4 import BeautifulSoup
import pandas as pd
import time

def request_url(url):
    # 设置请求头，模拟浏览器访问
    headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                          'Chrome/88.0.4324.146 Safari/537.36',
        }
    try:
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        return response.text
    except requests.RequestException as e:
        logging.error(f"Failed to retrieve data from {url}. Error: {e}")
        return None


def parse_comment(item):
    '''解析每一条评论'''
    comment = {}
    # 用户名称
    username = item.select_one('.comment-info a') # a是一个标签选择器，用于选取<a>标签
    comment['用户名称'] = username.text.strip() if username else ''
    # print(comment['用户名称'])

    # 评分
    score = item.select_one('span[class^="allstar"]')
    if score:
        score_class = score.get('class', [''])[0]
        # print(f"score_class: {score_class}")
        comment['评分'] = int(score_class.replace('allstar', ''))
    else:
        comment['评分'] = '无评分'

    # 评论时间
    time = item.select_one('.comment-time')
    comment['评论时间'] = time.text

    # 用户地址
    location = item.select_one('.comment-location')
    comment['用户地址'] = location.text.strip()

    # 评论内容
    content = item.select_one('.comment-content')
    comment['评论内容'] = content.text.strip()

    return comment


def get_comments(text):
    '''获取单页评论数据'''
    if text:
        soup = BeautifulSoup(text, 'lxml')
        items = soup.select('.comment-item')
        return [parse_comment(item) for item in items]

def main():
    all_comments = []
    for i in range(0, 6):
        if i == 0:
            url = "https://movie.douban.com/subject/34780991/comments"
        else:
            url = f"https://movie.douban.com/subject/34780991/comments?start={i * 20}&limit=20&status=P&sort=new_score"
        print(f"正在爬取：{url}")
        text = request_url(url)
        comments = get_comments(text)
        # print(comments)
        all_comments.extend(comments)
        # print(all_comments)
        time.sleep(3)

    # 保存数据，按指定顺序排列字段
    columns_order = ['用户名称', '评分', '评论时间', '用户地址', '评论内容']
    df = pd.DataFrame(all_comments, columns=columns_order)
    df.to_excel('豆瓣电影评论数据.xlsx', index=False)
    print('数据已保存到 豆瓣电影评论数据.xlsx')


if __name__ == '__main__':
    main()
